﻿/*
   Copyright 2012 University of Southampton

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Security.Principal;
using System.Management;
using System.IO;

namespace uk.ac.soton.ses.DAL
{
    public partial class MaterialsMetadataContext
    {
        /// <summary>
        /// Finds a matching experiment by its ID. May return null if the experiment cannot be found. It is expected that the caller checks for null
        /// </summary>
        /// <param name="id">The ID of the experiment</param>
        /// <returns>The matching experiment or null if the experiment cannot be found</returns>
        public Experiment GetExperimentByID(int id)
        {
            Experiment experiment = this.Experiments.FirstOrDefault(exp => exp.ID == id);
            //if (experiment.HasPermission())
            //{
            //}
            //FirstOrDefault may return null if the experiment cannot be found. It is expected that the caller checks for null
            return experiment;
        }

        /// <summary>
        /// Finds a matching experiment data file by its ID. May return null if the id cannot be found. It is expected that the caller checks for null
        /// </summary>
        /// <param name="id">The ID of the data file</param>
        /// <returns>The matching data file or null if it cannot be found</returns>
        public ExperimentDataFile GetExperimentDataFileByID(int id)
        {
            ExperimentDataFile experimentDataFile = this.ExperimentDataFiles.FirstOrDefault(df => df.ID == id);
            //FirstOrDefault may return null if the experiment cannot be found. It is expected that the caller checks for null
            return experimentDataFile;
        }

        /// <summary>
        /// Find data files under an experiment container
        /// </summary>
        /// <param name="experimentid">The experiment ID</param>
        /// <param name="includeDeleted">Whether deleted data files should be included in the results.</param>
        /// <returns>A set of data files</returns>
        public IQueryable<ExperimentDataFile> GetExperimentDataFilesByExperimentID(int experimentid, bool includeDeleted)
        {
            IQueryable<ExperimentDataFile> dataFiles = this.ExperimentDataFiles.Where(df => df.Experiment.ID == experimentid);
            if (!includeDeleted)
            {
                dataFiles = dataFiles.Where(df => !df.IsDeleted);
            }
            return dataFiles;
        }

        /// <summary>
        /// Find experiments under a collection
        /// </summary>
        /// <param name="collectionID">The collection ID</param>
        /// <param name="includeDeleted">Whether to return deleted experiments</param>
        /// <returns>A set of experiments in the container</returns>
        public IQueryable<Experiment> GetExperimentsByCollectionID(int collectionID, bool includeDeleted)
        {
            IQueryable<Experiment> experimentsInCollection =
                this.ExperimentCollectionExperimentMaps.Where(map => map.ExperimentCollectionID == collectionID)
                .Select(map => map.Experiment);

            if (!includeDeleted)
            {
                return experimentsInCollection.Where(exp => !exp.IsDeleted);
            }
            else
            {
                return experimentsInCollection;
            }

        }

        /// <summary>
        /// Returns an experiment collection
        /// </summary>
        /// <param name="collectionID">The ID of the collection to find</param>
        /// <returns>The experiment collection</returns>
        public ExperimentCollection GetExperimentCollectionByID(int collectionID)
        {
            return this.ExperimentCollections.FirstOrDefault(coll => coll.ID == collectionID);
        }

        /// <summary>
        /// Creates a relationship between an experiment and an experiment type
        /// </summary>
        /// <param name="experimentid">The ID of the experiment</param>
        /// <param name="typeid">The ID of the experiment type</param>
        /// <returns>The relationship</returns>
        public ExperimentTypeExperimentTemplateMap CreateTypeExperimentLink(int experimentid, int typeid)
        {
            ExperimentTypeExperimentTemplateMap newMap = new ExperimentTypeExperimentTemplateMap();
            newMap.Experiment = this.Experiments.First(exp => exp.ID == experimentid);
            newMap.ExperimentType = this.ExperimentTypes.First(type => type.ID == typeid);
            this.AddToExperimentTypeExperimentTemplateMaps(newMap);
            this.SaveChanges();
            return newMap;
        }

        /// <summary>
        /// Creates an experiment type, but throws an exception if it already exists
        /// </summary>
        /// <param name="name">The name of the new type</param>
        /// <returns>The new experiment type</returns>
        /// <exception cref="NotSupportedException">Thrown if the exception already exists</exception>
        public ExperimentType CreateExperimentType(string name)
        {
            if (this.ExperimentTypes.Count(type => type.Name == name) == 0)
            {
                ExperimentType newType = new ExperimentType();
                newType.Name = name;
                this.AddToExperimentTypes(newType);
                this.SaveChanges();
                return newType;
            }
            else
            {
                throw new NotSupportedException("Can't create type with existing name");
            }
        }

        /// <summary>
        /// Deletes the relationship between an experiment and experiment type
        /// </summary>
        /// <param name="experimentid">The ID of the experiment</param>
        /// <param name="typeid">The ID of the experiment type</param>
        public void DeleteTypeExperimentLink(int experimentid, int typeid)
        {
            //Find the relationship
            ExperimentTypeExperimentTemplateMap existingMap =
                this.ExperimentTypeExperimentTemplateMaps
                .FirstOrDefault(map => map.ExperimentTypeID == typeid && map.TemplateExperimentID == experimentid);
            if (existingMap != null)
            {
                //Delete the relationship
                this.DeleteObject(existingMap);
            }
        }

        /// <summary>
        /// Adds an experiment parameter to an experiment
        /// </summary>
        /// <param name="experimentid">The ID of the experiment</param>
        /// <param name="experimentParameter">The experiment parameter</param>
        public void AddParameterToExperiment(int experimentid, ExperimentParameter experimentParameter)
        {
            Experiment exp = this.GetExperimentByID(experimentid);
            if (exp != null)
            {
                exp.ExperimentParameters.Add(experimentParameter);
            }
            else
            {
                throw new KeyNotFoundException("Experiment " + experimentid + " not found");
            }
        }

        /// <summary>
        /// Looks up an experiment parameter with its ID
        /// </summary>
        /// <param name="experimentParameterID">The ID of the parameter</param>
        /// <returns>The parameter, or null if it cannot be found</returns>
        public ExperimentParameter GetExperimentParameterByID(int experimentParameterID)
        {
            ExperimentParameter experimentParameter = this.ExperimentParameters.FirstOrDefault(param => param.ID == experimentParameterID);
            return experimentParameter;
        }

        /// <summary>
        /// Returns a list of child parameters of a specified parameter
        /// </summary>
        /// <param name="parentParameterID">The ID of the parent parameter</param>
        /// <returns>A collection of child parameters</returns>
        public IQueryable<ExperimentParameter> GetExperimentChildParameters(int parentParameterID)
        {
            IQueryable<ExperimentParameter> childParams =
                this.ExperimentParameterLinks
                .Where(link => link.ExperimentParameterParent.ID == parentParameterID)
                .Select(link => link.ExperimentParameterLinked);
            return childParams;
        }

        /// <summary>
        /// Return a list of parameters for an experiment, but not the nested ones
        /// </summary>
        /// <param name="experimentID">The ID of the experiment to use</param>
        /// <param name="includeSystem">Whether to include system parameters</param>
        /// <param name="parameterName">Optional filter on the parameter name</param>
        /// <returns>A collection of top level parameters</returns>
        public IEnumerable<ExperimentParameter> GetExperimentTopLevelParameters(int experimentID, bool includeSystem, string parameterName = null)
        {
            IQueryable<ExperimentParameter> experimentParams =
                    this.ExperimentParameters
                    .Where(param =>
                        param.Experiment.ID == experimentID
                            //Filter out parameters that are listed as child parameters in the links table
                        && !this.ExperimentParameterLinks.Any(link => link.LinkedParameterID == param.ID));
            if (!string.IsNullOrEmpty(parameterName))
            {
                experimentParams = experimentParams.Where(param => param.Name == parameterName);
            }

            if (!includeSystem)
            {
                return experimentParams.AsEnumerable().Where(param => !param.IsSystemParameter);
            }
            else
            {
                return experimentParams;
            }
        }

        /// <summary>
        /// Retrieve parameters from an experiment, filtering by parameter name, system and position.
        /// </summary>
        /// <param name="experimentID">The ID of the data set</param>
        /// <param name="includeSystem">Whether to include parameters marked as system</param>
        /// <param name="parameterName">Optionally find parameters with a specific name</param>
        /// <param name="position">Whether to find parameters with a specific position</param>
        /// <returns>An IEnumerable of parameters matching the requirements</returns>
        public IEnumerable<ExperimentParameter> GetExperimentParameters(int experimentID, bool includeSystem, string parameterName = null,int? position = null)
        {
            IQueryable<ExperimentParameter> experimentParams =
                this.ExperimentParameters
                .Where(param =>
                    param.Experiment.ID == experimentID);
            if (!string.IsNullOrEmpty(parameterName))
            {
                experimentParams = experimentParams.Where(param => param.Name == parameterName);
            }
            if (position != null)
            {
                experimentParams = experimentParams.Where(param => param.Position == position);
            }
            if (!includeSystem)
            {
                return experimentParams.AsEnumerable().Where(param => !param.IsSystemParameter);
            }
            else
            {
                return experimentParams;
            }
        }

        /// <summary>
        /// Return experiments that a user has access to
        /// </summary>
        /// <param name="si">The user's security identifier</param>
        /// <param name="accesslevel">The access level required by the user</param>
        /// <returns>An IEnumerable of datasets the user can access</returns>
        public IEnumerable<Experiment> GetExperimentsForUser(SecurityIdentifier si, HDCExperimentAccessLevels accesslevel)
        {
            //Look up user
            User user = this.GetUserOrCreate(si);

            //Get this user's experiments using the stored procedure
            IEnumerable<Experiment> readableExperiments = this.spGetUsersExperiments(user.ID, (int)accesslevel);

            return readableExperiments;
        }

        /// <summary>
        /// Returns experiments that have a parameter defined on them, optionally with a specific value
        /// </summary>
        /// <param name="parameterName">The parameter name that the data set must have</param>
        /// <param name="parameterValue">The parameter value that the parameter must be set to</param>
        /// <returns>An IEnumerable of data sets that have a matching parameter</returns>
        public IEnumerable<Experiment> GetExperimentsWithMatchingParameter(string parameterName, string parameterValue = null)
        {
            IQueryable<ExperimentParameter> query = this.ExperimentParameters.Where(param => param.Name == parameterName);

            if (!(parameterValue == null))
            {
                query = query.Where(param => param.Value == parameterValue);
            }

            IEnumerable<Experiment> matchingExperiments = query.Select(param => param.Experiment);
            return matchingExperiments;
        }

        /// <summary>
        /// Finds data sets linked to from the specified data set (specified > referenced)
        /// </summary>
        /// <param name="experimentID">The ID of a data set to search for links from</param>
        /// <returns>An IEnumerable of data sets that are linked to</returns>
        public IEnumerable<ExperimentLink> GetReferencedExperimentLinks(int experimentID)
        {
            IEnumerable<ExperimentLink> referencedExperiments =
                 this.ExperimentLinks.Include("ExperimentLinked").Include("ExperimentParent")
                .Where(link => link.ExperimentParent.ID == experimentID
                    //Exclude two way links
                    && this.ExperimentLinks.Count(link2 => link2.ExperimentParent.ID == link.ExperimentLinked.ID && link2.ExperimentLinked.ID == link.ExperimentParent.ID) == 0);
            return referencedExperiments.AsEnumerable();
        }

        /// <summary>
        /// Find data sets that link to the specified data set (referencing > specified)
        /// </summary>
        /// <param name="experimentID">The ID of a data set to search against to find referencing data sets</param>
        /// <returns>An IEnumerable of data sets that link to specified data set</returns>
        public IEnumerable<ExperimentLink> GetReferencingExperimentLinks(int experimentID)
        {
            IEnumerable<ExperimentLink> referencedExperiments =
                 this.ExperimentLinks.Include("ExperimentLinked").Include("ExperimentParent")
                .Where(link => link.ExperimentLinked.ID == experimentID
                    //Exclude two way links
                    && this.ExperimentLinks.Count(link2 => link2.ExperimentParent.ID == link.ExperimentLinked.ID && link2.ExperimentLinked.ID == link.ExperimentParent.ID) == 0);
            return referencedExperiments.AsEnumerable();
        }

        /// <summary>
        /// Find data sets that have a two-way link to/from the specified data set.
        /// </summary>
        /// <param name="experimentID">The ID of a data set to search against to find other data sets that have a two-way link</param>
        /// <returns>An IEnumerable of data sets that have a two-way link with specified data set</returns>
        public IEnumerable<ExperimentLink> GetTwoWayLinkedExperiments(int experimentID)
        {
            IQueryable<ExperimentLink> referencedExperiments =
                   this.ExperimentLinks.Include("ExperimentLinked").Include("ExperimentParent")
                  .Where(link => link.ExperimentParent.ID == experimentID
                      //Ensure we get two way links
                        && this.ExperimentLinks.Count(link2 => link2.ExperimentParent.ID == link.ExperimentLinked.ID && link2.ExperimentLinked.ID == link.ExperimentParent.ID) == 1);
            return referencedExperiments.AsEnumerable();
        }

        /// <summary>
        /// Find data sets that are linked to by the specified data set (either one-way or two way)
        /// </summary>
        /// <param name="experimentID">The ID of the data set whose links are to be found</param>
        /// <returns>An IQueryable of data sets that are linked to</returns>
        public IQueryable<Experiment> GetOneWayAndTwoWayLinkedExperiments(int experimentID)
        {
            IQueryable<Experiment> linkedExperiments =
                this.ExperimentLinks
                .Where(link => link.ExperimentParent.ID == experimentID)
                .Select(link => link.ExperimentLinked);
            return linkedExperiments;
        }

        /// <summary>
        /// Create a link between two data sets.
        /// </summary>
        /// <param name="parentExperimentID">ID of data set to create link from</param>
        /// <param name="linkedExperimentID">ID of data set to create link to</param>
        /// <param name="saveChanges">Whether to update the database context or wait for SaveChanges to be called manually.</param>
        /// <returns>true if a link is created, false if the link exists already or experiment IDs are invalid</returns>
        public bool CreateExperimentLink(int parentExperimentID, int linkedExperimentID, bool saveChanges)
        {
            Experiment experimentToLinkTo = this.GetExperimentByID(linkedExperimentID);
            Experiment parentExperiment = this.GetExperimentByID(parentExperimentID);

            //Check we have all the data we need
            if (experimentToLinkTo != null && parentExperiment != null)
            {
                //Check we don't already have a link
                int existingLinks = this.ExperimentLinks.Count(link => link.ExperimentParent.ID == parentExperiment.ID && link.ExperimentLinked.ID == experimentToLinkTo.ID);
                if (existingLinks == 0)
                {
                    //Add a new link
                    ExperimentLink newLink = new ExperimentLink();
                    newLink.ExperimentParent = parentExperiment;
                    newLink.ExperimentLinked = experimentToLinkTo;
                    this.AddToExperimentLinks(newLink);
                    if (saveChanges) this.SaveChanges();

                    return true;
                }
                else
                {
                    return false;
                }
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// Removes a directional link between two data sets.
        /// </summary>
        /// <param name="parentExperimentID">ID of the data set on the 'from' side of the link</param>
        /// <param name="linkedExperimentID">ID of the data set on the 'to' side of the link</param>
        /// <param name="saveChanges">Whether to update the database context or wait for SaveChanges to be called manually.</param>
        /// <returns>true if the link was found and was deleted successfully, otherwise false.</returns>
        public bool DeleteExperimentLink(int parentExperimentID, int linkedExperimentID, bool saveChanges)
        {
            ExperimentLink linkToDelete =
                this.ExperimentLinks.FirstOrDefault(link => link.ExperimentParent.ID == parentExperimentID && link.ExperimentLinked.ID == linkedExperimentID);

            //Check we have a record
            if (linkToDelete != null)
            {
                //Delete the link
                this.DeleteObject(linkToDelete);
                if (saveChanges) this.SaveChanges();
                return true;
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// Add an experiment to a collection.
        /// </summary>
        /// <param name="collectionID">The ID of the collection to add the data set to</param>
        /// <param name="experimentID">The ID of the data set to add to the collection</param>
        /// <param name="saveChanges">Whether to update the database context or wait for SaveChanges to be called manually.</param>
        /// <returns>true if the data set was successfully added, false if the collection or data set could not be found</returns>
        public bool AddExperimentToCollection(int collectionID, int experimentID, bool saveChanges)
        {
            ExperimentCollection experimentCollection = this.ExperimentCollections.FirstOrDefault(coll => coll.ID == collectionID);
            Experiment experiment = this.GetExperimentByID(experimentID);

            if (experiment != null&& experimentCollection != null)
            {
                //Only create a map if it doesn't already exist
                if (this.ExperimentCollectionExperimentMaps.Count(map => map.ExperimentID == experiment.ID && map.ExperimentCollectionID == experimentCollection.ID) == 0)
                {
                    ExperimentCollectionExperimentMap newMap = new ExperimentCollectionExperimentMap();
                    experiment.ExperimentCollectionExperimentMaps.Add(newMap);
                    experimentCollection.ExperimentCollectionExperimentMaps.Add(newMap);
                    if (saveChanges) this.SaveChanges();
                }
                return true;
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// Create an experiment collection. If a name is not provided, a default value is chosen.
        /// </summary>
        /// <param name="saveChanges">Whether to update the database context or wait for SaveChanges to be called manually.</param>
        /// <param name="newCollectionName">A name for the new collection. If not provided, a default value is chosen.</param>
        /// <returns>The ExperimentCollection object created</returns>
        public ExperimentCollection CreateExperimentCollection(bool saveChanges, string newCollectionName=null)
        {
            //Create a new Experiment collection
            ExperimentCollection newExperimentCollection = new ExperimentCollection();

            //Create a default name if not set
            if (newCollectionName == null)
            {
                newCollectionName = String.Format("Collection added {0}", DateTime.Now);
            }

            //Set up the new collection
            newExperimentCollection.Name = newCollectionName;

            //Save the new collection to the database
            this.AddToExperimentCollections(newExperimentCollection);

            if (saveChanges) this.SaveChanges();

            return newExperimentCollection;
        }

        /// <summary>
        /// Remove a data set from a collection.
        /// </summary>
        /// <param name="collectionID">The ID of the collection.</param>
        /// <param name="experimentID">The ID of the data set to remove from the collection.</param>
        /// <param name="saveChanges">Whether to update the database context or wait for SaveChanges to be called manually.</param>
        /// <returns>true if the experiment was found in the collection and was removed, false otherwise.</returns>
        public bool DeleteExperimentFromCollection(int collectionID, int experimentID, bool saveChanges)
        {
            //Find out which map between Experiment and ExperimentCollection is being deleted
            ExperimentCollectionExperimentMap mapToDelete = this.ExperimentCollectionExperimentMaps.FirstOrDefault(coll => coll.ExperimentCollectionID == collectionID && coll.ExperimentID == experimentID);

            //Check we have a record
            if (mapToDelete != null)
            {
                //Delete the map
                this.DeleteObject(mapToDelete);
                if (saveChanges) this.SaveChanges();
                return true;
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// Updates the name and description of an existing collection.
        /// </summary>
        /// <param name="collectionID">The ID of the collection to update.</param>
        /// <param name="newName">The new name for the collection.</param>
        /// <param name="newDescription">The new description for the collection.</param>
        /// <param name="saveChanges">Whether to update the database context or wait for SaveChanges to be called manually.</param>
        /// <returns>true if the collection was found and updated, otherwise false.</returns>
        public bool UpdateExperimentCollection(int collectionID, string newName, string newDescription, bool saveChanges)
        {
            //Load the collection from the database
            ExperimentCollection collection = this.ExperimentCollections.FirstOrDefault(coll => coll.ID == collectionID);

            //Check we have a matching collection
            if (collection != null)
            {
                //Modify the collection
                collection.Name = newName;
                collection.Description = newDescription;
                if(saveChanges) this.SaveChanges();

                return true;
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// Deletes an experiment parameter
        /// </summary>
        /// <param name="experimentParameterID">The ID of the experiment parameter</param>
        /// <param name="saveChanges">Whether to update the database context or wait for SaveChanges to be called manually.</param>
        /// <returns>True if successful and False if unsuccessful</returns>
        public bool DeleteExperimentParameter(int experimentParameterID, bool saveChanges)
        {
            ExperimentParameter paramToDelete = this.GetExperimentParameterByID(experimentParameterID);
            if (paramToDelete != null)
            {
                //Remove links to this parameter before deleting it
                foreach (ExperimentParameterLink link in this.ExperimentParameterLinks.Where(lnk => lnk.ExperimentParameterLinked.ID == paramToDelete.ID))
                {
                    this.DeleteObject(link);
                }
                foreach (ExperimentParameterLink link in this.ExperimentParameterLinks.Where(lnk => lnk.ExperimentParameterParent.ID == paramToDelete.ID))
                {
                    this.DeleteObject(link);
                }
                //Okay to now delete the parameter
                this.DeleteObject(paramToDelete);

                if (saveChanges) this.SaveChanges();

                return true;
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// Recursively deletes all child parameters beneath the specified parameter
        /// </summary>
        /// <param name="parentParameterID">The ID of the parent parameter to delete</param>
        /// <param name="includeSelf">Whether to delete the parent parameter also</param>
        /// <param name="saveChanges">Whether to update the database context or wait for SaveChanges to be called manually.</param>
        /// <returns>true if the deletion is successful, otherwise false</returns>
        public bool DeleteExperimentParameterTree(int parentParameterID, bool includeSelf, bool saveChanges)
        {
            ExperimentParameter parentParameter = this.GetExperimentParameterByID(parentParameterID);

            if (parentParameter == null)
            {
                return false;
            }

            bool success = true;

            //Delete any sub-parameters first by calling ourself recursively
            foreach (ExperimentParameterLink link in this.ExperimentParameterLinks.Where(lnk => lnk.ExperimentParameterParent.ID == parentParameter.ID))
            {
                success = this.DeleteExperimentParameterTree(link.LinkedParameterID, true, false);
                if (!success)
                {
                    return false;
                }
            }

            if (includeSelf)
            {
                //Then we can delete this parameter
                success = this.DeleteExperimentParameter(parentParameter.ID, false);
                if (!success)
                {
                    return false;
                }
            }

            if (saveChanges) this.SaveChanges();
            return true;
        }
    
        /// <summary>
        /// Create a parameter and associate it with an experiment
        /// </summary>
        /// <param name="experimentid">The experiment that is the parent of the parameter</param>
        /// <param name="name">The parameter name (compulsory)</param>
        /// <param name="value">The parameter value</param>
        /// <param name="unit">The parameter unit</param>
        /// <param name="type">The parameter type</param>
        /// <param name="iscompulsory">Whether the parameter is compulsory (value is required)</param>
        /// <returns>The new parameter or null if adding fails</returns>
        public ExperimentParameter AddParameterToExperiment(int experimentid, string name, string value, string unit, string type, bool iscompulsory)
        {
            Experiment experiment = this.GetExperimentByID(experimentid);
            if (experiment != null)
            {
                ExperimentParameter newParam =
                    new ExperimentParameter()
                    {
                        Name = name,
                        Value = value,
                        Unit = unit,
                        Type = type,
                        IsCompulsory = iscompulsory,
                    };

                experiment.ExperimentParameters.Add(newParam);
                this.AddToExperimentParameters(newParam);
                return newParam;
            }
            else
            {
                return null;
            }
        }

        /// <summary>
        /// Changes a parameter's data
        /// </summary>
        /// <param name="experimentParameterID">The ID of the parameter to change</param>
        /// <param name="name">The new name</param>
        /// <param name="value">The new value</param>
        /// <param name="unit">The new unit</param>
        /// <param name="type">The new type</param>
        /// <param name="isCompulsory">Whether the parameter is compulsory</param>
        /// <param name="position">An optional position for the parameter</param>
        /// <returns>Whether the update was successful</returns>
        public bool UpdateExperimentParameter(int experimentParameterID, string name, string value, string unit, string type, bool isCompulsory, int? position)
        {
            ExperimentParameter paramToUpdate = this.GetExperimentParameterByID(experimentParameterID);
            if (paramToUpdate != null)
            {
                paramToUpdate.Name = name;
                paramToUpdate.Value = value;
                paramToUpdate.Unit = unit;
                paramToUpdate.Type = type;
                paramToUpdate.IsCompulsory = isCompulsory;
                paramToUpdate.Position = position;
                return true;
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// Changes a data set's metadata
        /// </summary>
        /// <param name="experimentID">The ID of the data set to change</param>
        /// <param name="description">The new description</param>
        /// <param name="typeID">The new type</param>
        /// <returns>true if data set and type IDs are found and updated, otherwise false</returns>
        public bool UpdateExperiment(int experimentID, string description, int typeID)
        {
            Experiment experimentToUpdate = this.GetExperimentByID(experimentID);
            ExperimentType newType = this.ExperimentTypes.FirstOrDefault(type => type.ID == typeID);

            if (experimentToUpdate != null && newType != null)
            {
                experimentToUpdate.Description = description;
                experimentToUpdate.ExperimentType = newType;
                return true;
            }
            else
            {
                return false;

            }
        }

        /// <summary>
        /// Change a data file's metadata
        /// </summary>
        /// <param name="experimentdatafileID">The ID of the data file to change.</param>
        /// <param name="description">The new description.</param>
        /// <returns>true if the data file ID is found and updated, otherwise false</returns>
        public bool UpdateExperimentDataFile(int experimentdatafileID, string description)
        {
            ExperimentDataFile experimentDataFileToUpdate = this.GetExperimentDataFileByID(experimentdatafileID);
            if (experimentDataFileToUpdate != null)
            {
                experimentDataFileToUpdate.Description = description;
                return true;
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// Deletes all parent parameters of a child, making the parameter as a top-level parameter.
        /// Usually, a child will only have one parent. The database allows more but the interface does not.
        /// </summary>
        /// <param name="experimentParameterID">The ID of the child parameter.</param>
        /// <returns>A count of the number of parents deleted.</returns>
        public int DeleteExistingParameterParentLinks(int experimentParameterID)
        {
            IQueryable<ExperimentParameterLink> linksToDelete = this.ExperimentParameterLinks.Where(link => link.ExperimentParameterLinked.ID == experimentParameterID);
            int count = 0;
            foreach (ExperimentParameterLink linkToDelete in linksToDelete)
            {
                this.DeleteObject(linkToDelete);
                count++;
            }
            return count;
        }

        /// <summary>
        /// Make a specified parameter a child of another.
        /// </summary>
        /// <param name="parentParamID">The ID of the parameter that will become the new parent.</param>
        /// <param name="childParamID">The ID of the paramter that will become the child.</param>
        /// <param name="deleteExistingParents">A value of true will remove the link to the parent first and false will abort (if the child parameter already has an existing parent).</param>
        /// <returns>The Link data created in the database</returns>
        public ExperimentParameterLink CreateExperimentParameterLink(int parentParamID, int childParamID, bool deleteExistingParents)
        {
            //Look up parameters
            ExperimentParameter parentParam = this.GetExperimentParameterByID(parentParamID);
            ExperimentParameter childParam = this.GetExperimentParameterByID(childParamID);

            IQueryable<ExperimentParameterLink> existingLinks = this.ExperimentParameterLinks.Where(link => link.ExperimentParameterLinked.ID == childParamID);
            if (existingLinks.Count() > 0)
            {
                if (deleteExistingParents)
                {
                    //Remove from existing parents
                    this.DeleteExistingParameterParentLinks(childParamID);
                }
                else
                {
                    return null;
                }
            }

            //We can only continue if the parameter is not a parent of the new parent (which would create a loop).
            //This only checks one level at the moment
            IQueryable<ExperimentParameterLink> existingParents = this.ExperimentParameterLinks.Where(link =>
                    link.ExperimentParameterParent.ID == childParam.ID
                    && link.ExperimentParameterLinked.ID == parentParam.ID);
            if (existingParents.Count() == 0)
            {
                //Add to new parent
                ExperimentParameterLink newLink = new ExperimentParameterLink();
                newLink.ExperimentParameterLinked = childParam;
                newLink.ExperimentParameterParent = parentParam;
                this.AddToExperimentParameterLinks(newLink);
                return newLink;
            }
            else
            {
                return null;
            }

        }

        /// <summary>
        /// Returns the parent parameter
        /// </summary>
        /// <returns>The parameter's parent, or null if it is a top-level parameter</returns>
        public ExperimentParameter GetParentParameter(ExperimentParameter childParam)
        {
            //Load the parents if necessary
            if (!childParam.ExperimentParameterParents.IsLoaded)
            {
                childParam.ExperimentParameterParents.Load();
            }

            ExperimentParameterLink parentlink = childParam.ExperimentParameterParents.FirstOrDefault();
            if (parentlink != null)
            {
                //Load the link if necessary
                if (!parentlink.ExperimentParameterParentReference.IsLoaded)
                {
                    parentlink.ExperimentParameterParentReference.Load();
                }
                return parentlink.ExperimentParameterParent;
            }
            else
            {
                return null;
            }

        }

        /// <summary>
        /// Find peer parameters (that share the same parent or are at the top-level).
        /// </summary>
        /// <param name="parameterID">The ID of the parameter to use</param>
        /// <param name="includeSelf">Whether to include the parameter in the results.</param>
        /// <returns>An IEnumerable of peer parameters</returns>
        public IEnumerable<ExperimentParameter> GetExperimentParametersInSameLevel(int parameterID, bool includeSelf)
        {
            ExperimentParameter parameter = this.GetExperimentParameterByID(parameterID);
            ExperimentParameter parentParameter = this.GetParentParameter(parameter);
            IEnumerable<ExperimentParameter> peerParameters = null;
            if (parentParameter == null)
            {
                peerParameters = this.GetExperimentTopLevelParameters(parameter.Experiment.ID, false);
            }
            else
            {
                peerParameters = this.GetExperimentChildParameters(parentParameter.ID);
            }

            //Exclude the requested parameter if required
            if (!includeSelf)
            {
                peerParameters = peerParameters.Where(param => param.ID != parameterID);
            }

            return peerParameters;
        }

        /// <summary>
        /// Change position of a parameter to make it the first one among its peers.
        /// </summary>
        /// <param name="parameterIDToMove">The ID of the parameter to move</param>
        public void MoveParameterToLastInLevel(int parameterIDToMove)
        {
            //Find the parameter we will move
            ExperimentParameter parameterToMove = this.GetExperimentParameterByID(parameterIDToMove);

            //Find the peer parameters
            IEnumerable<ExperimentParameter> peerParameters = this.GetExperimentParametersInSameLevel(parameterIDToMove, false);
            
            //Sort them by their position and find the last one
            ExperimentParameter lastParameter = peerParameters.OrderBy(param => param.Position).LastOrDefault();

            //If there is no last parameter, the last position is 1
            if (lastParameter == null)
            {
                parameterToMove.Position = 1;
            }
            //If there is a last parameter, then we will move the parameter to 1 afterwards
            else
            {
                parameterToMove.Position = lastParameter.Position + 1;
            }
        }

        /// <summary>
        /// Change position of a parameter to make it the last one among its peers.
        /// </summary>
        /// <param name="parameterIDToMove">The ID of the parameter to move</param>
        public void MoveParameterToFirstInLevel(int parameterIDToMove)
        {
            //Find the parameter we will move
            ExperimentParameter parameterToMove = this.GetExperimentParameterByID(parameterIDToMove);

            //Find the peer parameters
            IEnumerable<ExperimentParameter> peerParameters = this.GetExperimentParametersInSameLevel(parameterIDToMove, false);

            //Sort them by their position and find the first one
            ExperimentParameter firstParameter = peerParameters.OrderBy(param => param.Position).FirstOrDefault();

            //If there is no first parameter, the first position is 1
            if (firstParameter == null)
            {
                parameterToMove.Position = 1;
            }
            //If there is already a first parameter, call ParameterReposition() to insert our parameter at the beginning
            else
            {
                this.ParameterReposition(parameterIDToMove, firstParameter.ID);
            }
        }

        /// <summary>
        /// Reorders the parameters in an experiment
        /// </summary>
        /// <param name="parameterIDToInsert">The ID of the parameter that is to be inserted</param>
        /// <param name="parameterIDToDemote">The ID of the parameter that is to be demoted</param>
        public void ParameterReposition(int parameterIDToInsert, int parameterIDToDemote)
        {
            //Get the parameters
            ExperimentParameter parameterToInsert = this.GetExperimentParameterByID(parameterIDToInsert);
            ExperimentParameter parameterToDemote = this.GetExperimentParameterByID(parameterIDToDemote);

            if (parameterToInsert == null || parameterToDemote == null)
            {
                throw new Exception("Parameter not found for repositioning");
            }

            //Are the parameters on the same experiment?
            if (parameterToInsert.Experiment.ID != parameterToDemote.Experiment.ID)
            {
                throw new Exception("Parameters are not from the same experiment");
            }

            //Check if parameterToInsert is in a linkedExperimentID field (i.e. it is a nested parameter) and get parentParameterID
            ExperimentParameter parameterToInsertParent = this.GetParentParameter(parameterToInsert);

            //Check if parameterIDToDemote is in a linkedExperimentID field (it is a nested parameter) and get its parentParameterID
            ExperimentParameter parameterToDemoteParent = this.GetParentParameter(parameterToDemote);
            bool sameLevel = true;

            //Compare the two parentExperimentIDs because, if they are different, we will have to edit the links, then do the reordering
            if (parameterToInsertParent != null && parameterToDemoteParent != null && parameterToInsertParent.ID != parameterToDemoteParent.ID)
            {
                this.CreateExperimentParameterLink(parameterToDemoteParent.ID, parameterToInsert.ID, true);
                parameterToInsertParent = parameterToDemoteParent;
                sameLevel=false;
            }
            else if (parameterToInsertParent != null && parameterToDemoteParent == null)
            {
                this.DeleteExistingParameterParentLinks(parameterToInsert.ID);
                parameterToInsertParent = null;
                sameLevel=false;
            }
            else if (parameterToInsertParent == null && parameterToDemoteParent != null)
            {
                this.CreateExperimentParameterLink(parameterToDemoteParent.ID, parameterToInsert.ID, true);
                parameterToInsertParent = parameterToDemoteParent;
                sameLevel=false;
            }

            IEnumerable<ExperimentParameter> parametersToDemote = null;
            if (parameterToInsertParent == null)
            {
                parametersToDemote = this.GetExperimentTopLevelParameters(parameterToInsert.Experiment.ID, false);
            }
            else
            {
                parametersToDemote = this.GetExperimentChildParameters(parameterToInsertParent.ID);
            }

            parametersToDemote = parametersToDemote.Where(param=>param.ID != parameterToInsert.ID);

            int? pos;
            if (parameterToInsert.Position == null || parameterToDemote.Position == null)
            {
                pos = 1;
                //We will have to renumber everything
                foreach (ExperimentParameter p in parametersToDemote.OrderBy(param=>param.Position))
                {
                    if (p.ID == parameterToDemote.ID)
                    {
                        parameterToInsert.Position = pos++;
                    }
                    p.Position = pos++;
                }
            }
            else if (sameLevel && parameterToInsert.Position < parameterToDemote.Position)
            {
                pos = parameterToInsert.Position;
                parametersToDemote =
                    parametersToDemote
                    .Where(param => param.Position > parameterToInsert.Position && param.Position<parameterToDemote.Position)
                    .OrderBy(param => param.Position);

                foreach (ExperimentParameter p in parametersToDemote)
                {
                    p.Position = pos++;
                }

                //Finally, set the inserted parameter's position
                parameterToInsert.Position = pos;
            }
            else
            {
                //Record parameterToDemote's position so we can set the inserted parameter's position later
                int? insertNewPos = parameterToDemote.Position;

                pos = parameterToDemote.Position;
                //Sort and filter the parameters (we only need to renumber parameters with a higher position)
                parametersToDemote =
                    parametersToDemote
                    .Where(param => param.Position >= parameterToDemote.Position)
                    .OrderBy(param => param.Position);

                //Iterate through all the parameters at this level with a position >= than parameterIDToDemote and increase its position
                foreach (ExperimentParameter p in parametersToDemote)
                {
                    p.Position = ++pos;
                }

                //Finally, set the inserted parameter's position
                parameterToInsert.Position = insertNewPos;
            }

            this.SaveChanges();
        }

        /// <summary>
        /// Returns a list of data sets whose names match the query string or contain data files or parameters which match the string
        /// </summary>
        /// <param name="query">The query string to use.</param>
        /// <param name="includeDeleted">Whether deleted data sets should be included in the search.</param>
        /// <returns>An IEnumerable containing a set of KeyValuePairs with the key being the ID of the data set and the value being a description of the item that was found.</returns>
        public IEnumerable<KeyValuePair<int, string>> FindExperimentsBySubstring(string query, bool includeDeleted)
        {
            //Find experiments
            IQueryable<Experiment> expquery =
                this.Experiments
                .Where(exp => exp.Name.Contains(query) || exp.Description.Contains(query));

            IQueryable<ExperimentDataFile> dfquery = 
                this.ExperimentDataFiles
                .Include("Experiment")
                .Where(df => df.Filename.Contains(query));

            IQueryable<ExperimentParameter> paramquery =
                this.ExperimentParameters
                .Include("Experiment")
                .Where(param => param.Name.StartsWith(query));

            if(!includeDeleted)
            {
                expquery = expquery.Where(exp => !exp.IsDeleted);
                dfquery = dfquery.Where(df=>!df.IsDeleted);
            }

            IEnumerable<KeyValuePair<int, string>> expdict =
                expquery
                .AsEnumerable()
                .Select(exp => new KeyValuePair<int, string>(exp.ID, exp.GetNameAndID(prepend:"Experiment ")));

            IEnumerable<KeyValuePair<int, string>> dfdict =
                dfquery
                .AsEnumerable()
                .Select(df => new KeyValuePair<int, string>(df.Experiment.ID, string.Format("File {2} [{0}|{1}]", df.Experiment.ID, df.Experiment.Name, df.Filename)));

            IEnumerable<KeyValuePair<int, string>> paramdict =
                paramquery
                .AsEnumerable()
                .Select(param => new KeyValuePair<int, string>(param.Experiment.ID, string.Format("Param {2} [{0}|{1}]", param.Experiment.ID, param.Experiment.Name, param.Name)));

            List<KeyValuePair<int, string>> result = new List<KeyValuePair<int, string>>();
            result.AddRange(expdict);
            result.AddRange(dfdict);
            result.AddRange(paramdict);
            return result;
        }

        /// <summary>
        /// Returns a list of data sets whose names begin with the query string
        /// </summary>
        /// <param name="name">The name of the data set to search for.</param>
        /// <param name="includeDeleted">Whether deleted data sets should be included in the search.</param>
        /// <returns>An IEnumerable containing a set of KeyValuePairs with the key being the ID of the data set and the value being a description of the data set found.</returns>
        public IEnumerable<KeyValuePair<int, string>> FindExperimentsByName(string name, bool includeDeleted)
        {
            IQueryable<Experiment> expquery =
                this.Experiments
                .Where(exp => exp.Name.StartsWith(name));
            if(!includeDeleted)
            {
                expquery=expquery.Where(exp=>!exp.IsDeleted);
            }
            IEnumerable<KeyValuePair<int, string>> expdict =
                expquery
                .AsEnumerable()
                .Select(exp => new KeyValuePair<int, string>(exp.ID, exp.GetNameAndID(idFirst: false)));
            return expdict;
        }

        /// <summary>
        /// Returns a list of data sets which contain parameters starting with the query string.
        /// </summary>
        /// <param name="name">The name of the parameter to search for.</param>
        /// <returns>An IEnumerable containing a set of KeyValuePairs with the key being the ID of the data set and the value being a description of the parameter found within the data set.</returns>
        public IEnumerable<KeyValuePair<int, string>> FindExperimentsByParameterName(string name)
        {
            Dictionary<int, string> dict = new Dictionary<int, string>();
            var matchingParams =
                this.ExperimentParameters
                .Include("Experiment")
                .Where(param => param.Name.StartsWith(name));
            foreach (ExperimentParameter param in matchingParams)
            {
                if (!dict.ContainsKey(param.Experiment.ID))
                {
                    dict.Add(param.Experiment.ID, string.Format("Param {2} [{0}|{1}]", param.Experiment.ID, param.Experiment.Name, param.Name));
                }
            }
            return dict.ToList();
        }

        /// <summary>
        /// Find the user's database record in the database by searching by its SID. If the user does not already exist, a new record will be created.
        /// </summary>
        /// <param name="si">The SecurityIdentifier of the user to find in the database.</param>
        /// <returns>The database record of the user.</returns>
        public User GetUserOrCreate(SecurityIdentifier si)
        {
            //Find it in the database
            IQueryable<User> users = this.Users.Where(usr => usr.UserID == si.Value);
            NTAccount ntaccount = si.Translate(typeof(NTAccount)) as NTAccount;

            switch (users.Count())
            {
                //User does not exist in database
                case 0:
                    //Lookup the account's properties using WMI
                    string[] accountparts = ntaccount.ToString().Split('\\');
                    string wmiQuery = String.Format("select * from Win32_UserAccount where domain='{0}' and name='{1}'", accountparts[0], accountparts[1]);
                    ManagementObjectSearcher wmiSearch = new ManagementObjectSearcher(wmiQuery);
                    ManagementObjectCollection wmiUsers = wmiSearch.Get();
                    if (wmiUsers.Count == 1)
                    {
                        ManagementObject wmiUser = wmiUsers.OfType<ManagementObject>().First();
                        User user = new User { Username = (string)wmiUser["Name"], UserID = (string)wmiUser["SID"], Name = (string)wmiUser["FullName"], Email = "none" };
                        this.AddToUsers(user);
                        //Save to ensure we get a new ID
                        this.SaveChanges();
                        return user;
                    }
                    else
                    {
                        throw new InvalidOperationException("Error loading user account properties:" + ntaccount.ToString());
                    }

                //User already exists, so just return it
                case 1:
                    return users.First();

                default:
                    throw new InvalidOperationException("Too many users found matching:" + ntaccount.ToString());

            }
        }

        /// <summary>
        /// Check the database to see whether a user belongs to the administrator group (with ID of zero).
        /// </summary>
        /// <param name="si">The SecurityIdentifier of the user.</param>
        /// <returns>true if the user is in the admin group, otherwise false.</returns>
        public bool UserIsAdmin(SecurityIdentifier si)
        {
            bool admin = false;

            //Check if the user is in the admin group
            admin |= this.Groups.Where(grp => grp.ID == 0).Any(grp => grp.UserGroupMaps.Any(map => map.User.UserID == si.Value));
            return admin;
        }

        /// <summary>
        /// Change the security in the database to grant a user access to a data set.
        /// </summary>
        /// <param name="userToGrant">The user to grant permission to in domain\user format.</param>
        /// <param name="experimentID">The ID of the data set.</param>
        /// <param name="writeAccess">true if write access is required, false if only read access is required.</param>
        public void GrantUserToExperiment(string userToGrant, int experimentID, bool writeAccess)
        {
            //Load the experiment
            Experiment experiment = this.GetExperimentByID(experimentID);

            if (userToGrant.Length > 0)
            {
                //Lookup the user, ignoring the exception if not found (this should be handled higher up)
                NTAccount acc = new NTAccount(userToGrant);
                SecurityIdentifier si = acc.Translate(typeof(SecurityIdentifier)) as SecurityIdentifier;
                User user = this.GetUserOrCreate(si);

                if (!this.UserAccesses.Where(ace => ace.Experiment.ID == experimentID && ace.User.ID == user.ID).Any())
                {
                    experiment.UserAccesses.Load();
                    experiment.UserAccesses.Add(new UserAccess() { User = user, WriteAccessGranted = writeAccess });
                }
            }
        }

        /// <summary>
        /// Change the owner of an experiment.
        /// </summary>
        /// <param name="si">The SecurityIdentifier of the new owner.</param>
        /// <param name="experimentID">The ID of the experiment to change.</param>
        public void SetExperimentOwner(SecurityIdentifier si, int experimentID)
        {
            //Lookup owner if there is one
            User owner = this.GetUserOrCreate(si);
            Experiment exp = this.GetExperimentByID(experimentID);
            if (owner != null && exp != null)
            {
                exp.User = owner;
            }
        }

        /// <summary>
        /// Change the owner of a data file.
        /// </summary>
        /// <param name="si">The SecurityIdentifier of the new owner.</param>
        /// <param name="datafileID">The ID of the data file to change.</param>
        public void SetExperimentDataFileOwner(SecurityIdentifier si, int datafileID)
        {
            //Lookup owner if there is one
            User owner = this.GetUserOrCreate(si); // context.Users.FirstOrDefault(usr => usr.UserID == newExperiment.FileSystemUserID);
            ExperimentDataFile df = this.GetExperimentDataFileByID(datafileID);
            if (owner != null && df != null)
            {
                df.User = owner;
            }
        }

        /// <summary>
        /// Remove a user's access to a data set.
        /// </summary>
        /// <param name="dbUserID">The ID of the user according to the database's ID column/key field</param>
        /// <param name="experimentID">The ID of the data set to change.</param>
        public void DeleteUserAccessToExperiment(int dbUserID, int experimentID)
        {
            Experiment experiment = this.GetExperimentByID(experimentID);
            experiment.UserAccesses.Load();

            IQueryable<UserAccess> userACEs = this.Experiments.Where(exp => exp.ID == experimentID).SelectMany(exp => exp.UserAccesses.Where(ace => ace.User.ID == dbUserID));
            foreach (UserAccess ace in userACEs)
            {
                this.DeleteObject(ace);
            }
        }

        /// <summary>
        /// Change a user's access from read/write to read only, or vice versa.
        /// </summary>
        /// <param name="dbUserID">The ID of the user according to the database's ID column/key field.</param>
        /// <param name="experimentID">The ID of the data set to change.</param>
        public void ToggleWriteAccessToExperiment(int dbUserID, int experimentID)
        {
            Experiment experiment = this.GetExperimentByID(experimentID);
            experiment.UserAccesses.Load();

            IQueryable<UserAccess> userACEs = this.UserAccesses.Where(ace => ace.Experiment.ID == experimentID && ace.User.ID == dbUserID);
            foreach (UserAccess ace in userACEs)
            {
                ace.WriteAccessGranted = !ace.WriteAccessGranted;
            }
        }

        /// <summary>
        /// Duplicates all of the parameters between experiments. Useful for copying parameters from a template, but could be used for other reasons.
        /// </summary>
        /// <param name="sourceExperimentID">The ID of the source experiment.</param>
        /// <param name="destExperimentID">The ID of the destination experiment.</param>
        /// <param name="copySystem">Whether to copy the system parameters as well.</param>
        public void CopyParametersBetweenExperiments(int sourceExperimentID, int destExperimentID, bool copySystem)
        {
            Experiment sourceExperiment = this.GetExperimentByID(sourceExperimentID);
            Experiment destExperiment = this.GetExperimentByID(destExperimentID);
            WindowsIdentity identity = System.Security.Principal.WindowsIdentity.GetCurrent();
            if (sourceExperiment != null && destExperiment != null)
            {
                if (sourceExperiment.HasPermission(identity, this) && destExperiment.HasPermission(identity, this))
                {
                    sourceExperiment.CopyParametersToExperiment(destExperiment, copySystem, this);
                }
                else
                {
                    throw new ExperimentAccessDeniedException("Access denied to experiment");
                }
            }
        }

        /// <summary>
        /// Create a new template from an experiment, copying across all of the parameters and setting the type correctly.
        /// </summary>
        /// <param name="newOwner">The owner of the experiment</param>
        /// <param name="sourceExperimentID">The ID of the data set to find and use as the source for the template.</param>
        /// <param name="newName">The name for the new template</param>
        /// <param name="copySystem">Whether to copy HDC system parameters</param>
        /// <returns>A new experiment with the specified name</returns>
        public Experiment CreateTemplateFromExperiment(SecurityIdentifier newOwner, int sourceExperimentID, string newName, bool copySystem)
        {
            Experiment sourceExperiment = this.GetExperimentByID(sourceExperimentID);
            if (sourceExperiment != null)
            {
                return sourceExperiment.CopyToNewTemplate(newOwner, newName, copySystem, this);
            }
            else
            {
                throw new ExperimentLoadException("Experiment not found");
            }
        }

        /// <summary>
        /// Generate a selection of experiments that have the same type as another experiment
        /// </summary>
        /// <param name="experimentID">The experiment to compare against</param>
        /// <param name="count">The number of experiments to return</param>
        /// <param name="findDeleted">Whether to return deleted experiments in the results</param>
        /// <returns>A list of experiments that have the same type as the requested experiment</returns>
        public List<Experiment> GetExperimentsOfSameType(int experimentID, int count = 5, bool findDeleted = false)
        {
            Experiment selectedExperiment = this.GetExperimentByID(experimentID);
            if (selectedExperiment != null)
            {
                //Find experiments of same type as selectedExperiment, excluding the selectedExperiment
                IQueryable<Experiment> sameType = this.Experiments.Where(exp => exp.ExperimentType.ID == selectedExperiment.ExperimentType.ID && exp.ID != experimentID);

                //Filter out deleted experiments if requested
                if (!findDeleted)
                {
                    sameType = sameType.Where(exp => !exp.IsDeleted);
                }

                //The GetRandomSelection function requires an ordered queryable
                IOrderedQueryable<Experiment> sameTypeOrderedByID = sameType.OrderBy(exp => exp.ID);

                //We only want a selection of records, so generate a random selection
                List<Experiment> result = this.GetRandomSelection(sameTypeOrderedByID, count);
                
                return result;
            }
            else
            {
                throw new ExperimentLoadException("Experiment not found");
            }
        }

        /// <summary>
        /// Suggest experiments that relate to a selected experiment
        /// </summary>
        /// <param name="experimentID">The selected experiment's ID</param>
        /// <param name="count">The number of suggestions to return</param>
        /// <param name="findDeleted">Whether to include deleted experiments</param>
        /// <returns>A list of suggested experiments</returns>
        public List<Experiment> GetRelatedExperiments(int experimentID, int count = 5, bool findDeleted = false)
        {
            Experiment selectedExperiment = this.GetExperimentByID(experimentID);
            if (selectedExperiment != null)
            {
                //Find experiments that link to the same experiment as we link to
                //We're basically doing this:
                // select ParentExperimentID from ExperimentLinks where LinkedExperimentID in
                //  (select ParentExperimentID from ExperimentLinks where LinkedExperimentID=16)

                //Find experiments where selectedExperiment is the linked experiment of another, and then lookup what experiments link to that
                //"Which experiments link to the ones that link to us?" - RECOMMENDED > LINKED > OUREXP
                IQueryable<Experiment> linkedExperiments1 =
                    this.ExperimentLinks
                    .Where(link1 =>
                        //This finds links where the linked experiment matches our selected experiment
                        this.ExperimentLinks.Where(link2 => link2.LinkedExperimentID == selectedExperiment.ID)
                            //This uses link1 (all links) to find links where the linked experiment is the same as the parent of link2
                            .Any(link2 => link1.LinkedExperimentID == link2.ParentExperimentID)
                    )
                    //This then selects the parent experiment of the matching links
                    .Select(link => link.ExperimentParent)
                    //Ensure we don't already link to it
                    .Where(exp => !this.ExperimentLinks.Any(link => link.ParentExperimentID == selectedExperiment.ID && link.LinkedExperimentID == exp.ID));

                //Find links where the selectedExperiment links to another, and then lookup what experiments that one links to
                //"Which experiments do the linked experiments link to?" OUREXP > LINKED > RECOMMENDED
                IQueryable<Experiment> linkedExperiments2 =
                    this.ExperimentLinks
                    .Where(link1 =>
                        //This finds links where the parent experiment matches our selected experiment
                        this.ExperimentLinks.Where(link2 => link2.ParentExperimentID == selectedExperiment.ID)
                            //This searches all links (link1) to find links where the parent experiment is the same as the linked experiment of link2
                            .Any(link2 => link1.ParentExperimentID==link2.LinkedExperimentID))
                    //This then selects the linked experiment of the matching links
                    .Select(link => link.ExperimentLinked)
                    //Ensure we don't already link to it
                    .Where(exp => !this.ExperimentLinks.Any(link => link.ParentExperimentID == selectedExperiment.ID && link.LinkedExperimentID == exp.ID));

                //"Which experiments link to our linked experiment?" OUREXP > LINKED < RECOMMENDED
                IQueryable<Experiment> linkedExperiments3 =
                    this.ExperimentLinks.Where(link1 =>
                        //all the links which have a parent experiment id of link2.linkedid?
                        this.ExperimentLinks.Where(link2 => link2.ParentExperimentID == selectedExperiment.ID)
                        .Any(link2 => link1.LinkedExperimentID == link2.LinkedExperimentID && link1.ParentExperimentID != selectedExperiment.ID))
                    .Select(link => link.ExperimentParent);
                    //    .Select(link2=>link2.ExperimentLinked).Select(link3=>link3.ExperimentParents)
                    //    .Any(link2 => link1.LinkedExperimentID == link2.LinkedExperimentID && link1.ParentExperimentID!=selectedExperiment.ID))
                    //.Where(link=>link.ParentExperimentID!=selectedExperiment.ID && link.LinkedExperimentID!=
                    //.Select(link=>link.ExperimentParent)
                    //.Where(exp=>;
                    //.Select(link=>this.ExperimentLinks.Where(link2=>link2.ParentExperimentID==link.LinkedExperimentID))

                //Filter out the deleted experiments if requested
                if (!findDeleted)
                {
                    linkedExperiments1 = linkedExperiments1.Where(exp => !exp.IsDeleted);
                    linkedExperiments2 = linkedExperiments2.Where(exp => !exp.IsDeleted);
                    linkedExperiments3 = linkedExperiments3.Where(exp => !exp.IsDeleted);
                }

                //Combine the two queries, so we can filter it
                IOrderedQueryable<Experiment> linkedExperiments = linkedExperiments1.Union(linkedExperiments2).Union(linkedExperiments3).OrderBy(exp => exp.ID);

                //Get a random selection of experiments of the number requested
                List<Experiment> result = this.GetRandomSelection(linkedExperiments, count);
                return result;
            }
            else
            {
                throw new ExperimentLoadException("Experiment not found");
            }
        }

        /// <summary>
        /// Give a query of Experiments, this selects some at random
        /// </summary>
        /// <param name="expQuery">A query that returns some experiments</param>
        /// <param name="numRequired">The maximum number of experiments required</param>
        /// <returns>A list of experiments selected at random</returns>
        private List<Experiment> GetRandomSelection(IOrderedQueryable<Experiment> expQuery, int numRequired)
        {
            int dbrecordcount = expQuery.Count();
            if (dbrecordcount <= numRequired)
            {
                return expQuery.ToList();
            }
            else
            {
                //Based on the number required, we generated the indexes of some random records
                List<int> rands = HDCSharedFunctions.GetUniqueRandomIntegers(numRequired, dbrecordcount, 0, true);
                List<Experiment> result = new List<Experiment>();
                foreach (int i in rands)
                {
                    //This is a horrible way to do it (1 round trip to database for each record).
                    //Later versions of entity framework permit: 'result.Where((exp,index)=>rands.Contains(index))' which would help perform this as one operation
                    //Even better would be to ask the database to call NEWID to generate a GUID and sort by that - again, not possible in EF3.5
                    result.AddRange(expQuery.Skip(i).Take(1));
                }
                return result;
            }
        }

        /// <summary>
        /// This creates a data set on the file system and waits for the watcher to identify it, returning its database record.
        /// </summary>
        /// <param name="experimentName">The name of the new data set.</param>
        /// <param name="newOwner">The SecurityIdentifier of the new owner of the data set.</param>
        /// <returns>The database record representing the new data set.</returns>
        /// <exception cref="System.IO.DirectoryNotFoundException">Thrown if the file system watcher's base path is not found.</exception>
        /// <exception cref="System.IO.IOException">Thrown if directory already exists.</exception>
        /// <exception cref="System.IO.DirectoryNotFoundException">Thrown if data set is not discovered by file system watcher within 10 seconds.</exception>
        public Experiment AddExperimentToFileSystem(string experimentName, SecurityIdentifier newOwner)
        {
            //get base path
            string parentFolder = HDCConfiguration.Instance.ConfigurationOptions.FileSystemWatcherOptions.LocalPath;
            int basePathDepth = HDCConfiguration.Instance.ConfigurationOptions.FileSystemWatcherOptions.BasePathDepthInWatchFolder;
            for (int i = 0; i < basePathDepth; i++)
            {
                parentFolder = Path.Combine(parentFolder, "web");
            }
            if (!Directory.Exists(parentFolder))
            {
                throw new DirectoryNotFoundException("Base path for creating experiment not found");
            }

            string experimentFolderPath = Path.Combine(parentFolder,experimentName);
            if(Directory.Exists(experimentFolderPath))
            {
                throw new IOException("Directory exists");
            }
            else
            {
                Directory.CreateDirectory(experimentFolderPath);
            }

            //Wait for the record to appear in the database so we can return the file
            int check = 1;
            Experiment newExp = null;
            while (check <= 10 && newExp == null)
            {
                newExp = this.Experiments.FirstOrDefault(exp => !exp.IsDeleted && exp.DefaultBasePath == experimentFolderPath);

                //If we didn't find a record yet, wait for a bit and then check again
                if (newExp == null)
                {
                    System.Threading.Thread.Sleep(1000);
                    check++;
                }
            }

            if (newExp == null)
            {
                string message = string.Format("Could not find folder {0}", experimentFolderPath);
                throw new DirectoryNotFoundException(message);
            }

            //Grant the owner permission
            this.GrantUserToExperiment(((NTAccount)newOwner.Translate(typeof(NTAccount))).ToString(), newExp.ID, true);
            this.SaveChanges();

            //Return the experiment
            return newExp;

        }
    }
}